{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 导入工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 设定数据列名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "CONTINUOUS_COLUMNS =  [\"I\"+str(i) for i in range(1,14)] # 1-13 inclusive\n",
    "CATEGORICAL_COLUMNS = [\"C\"+str(i) for i in range(1,27)] # 1-26 inclusive\n",
    "LABEL_COLUMN = [\"clicked\"]\n",
    "\n",
    "TRAIN_DATA_COLUMNS = LABEL_COLUMN + CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义处理格式处理函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input function configured\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 400\n",
    "\n",
    "def generate_input_fn(filename, batch_size=BATCH_SIZE):\n",
    "    def _input_fn():\n",
    "        filename_queue = tf.train.string_input_producer([filename])\n",
    "        reader = tf.TextLineReader()\n",
    "        # Reads out batch_size number of lines\n",
    "        key, value = reader.read_up_to(filename_queue, num_records=batch_size)\n",
    "        \n",
    "        # 1 int label, 13 ints, 26 strings\n",
    "        cont_defaults = [ [0] for i in range(1,14) ]\n",
    "        cate_defaults = [ [\" \"] for i in range(1,27) ]\n",
    "        label_defaults = [ [0] ]\n",
    "        column_headers = TRAIN_DATA_COLUMNS\n",
    "        # The label is the first column of the data.\n",
    "        # 如果有缺失值就用默认值代替\n",
    "        record_defaults = label_defaults + cont_defaults + cate_defaults\n",
    "\n",
    "        # Decode CSV data that was just read out. \n",
    "        # Note that this does NOT return a dict, \n",
    "        # so we will need to zip it up with our headers\n",
    "        columns = tf.decode_csv(\n",
    "            value, record_defaults=record_defaults)\n",
    "        \n",
    "        # all_columns is a dictionary that maps from column names to tensors of the data.\n",
    "        all_columns = dict(zip(column_headers, columns))\n",
    "        \n",
    "        # Pop and save our labels \n",
    "        # dict.pop() returns the popped array of values; exactly what we need!\n",
    "        labels = all_columns.pop(LABEL_COLUMN[0])\n",
    "        \n",
    "        # the remaining columns are our features\n",
    "        features = all_columns \n",
    "\n",
    "        # Sparse categorical features must be represented with an additional dimension. \n",
    "        # There is no additional work needed for the Continuous columns; they are the unaltered columns.\n",
    "        # See docs for tf.SparseTensor for more info\n",
    "        for feature_name in CATEGORICAL_COLUMNS:\n",
    "            features[feature_name] = tf.expand_dims(features[feature_name], -1)\n",
    "\n",
    "        return features, labels\n",
    "\n",
    "    return _input_fn\n",
    "\n",
    "print('input function configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立输入到DNN模型的特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wide/Sparse columns configured\n"
     ]
    }
   ],
   "source": [
    "# Sparse base columns：对于类别特征，用hash编码将其转化为稀疏列\n",
    "wide_columns = []# 用于Line模型的特征\n",
    "for name in CATEGORICAL_COLUMNS:\n",
    "    wide_columns.append(tf.contrib.layers.sparse_column_with_hash_bucket(\n",
    "            name, hash_bucket_size=1000))\n",
    "\n",
    "print('Wide/Sparse columns configured')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deep/continuous columns configured\n"
     ]
    }
   ],
   "source": [
    "# Continuous base columns：对于连续性特征，直接使用真实值。\n",
    "deep_columns = []# 用于DNN模型的特征\n",
    "for name in CONTINUOUS_COLUMNS:\n",
    "    deep_columns.append(tf.contrib.layers.real_valued_column(name))\n",
    "\n",
    "print('deep/continuous columns configured')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "WARNING:tensorflow:The default stddev value of initializer will change from \"1/sqrt(vocab_size)\" to \"1/sqrt(dimension)\" after 2017/02/25.\n",
      "wide and deep columns configured\n"
     ]
    }
   ],
   "source": [
    "# Embeddings for wide columns into deep columns：再将类别特征进行词嵌入模型，送入DNN模型。\n",
    "for col in wide_columns:\n",
    "    deep_columns.append(tf.contrib.layers.embedding_column(col, \n",
    "                                                           dimension=8))\n",
    "\n",
    "print('wide and deep columns configured')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立模型\n",
    "* **Deep**: Deep Neural Net Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model directory = models/model_DEEP_1560669305\n",
      "WARNING:tensorflow:From <ipython-input-9-b25ae821cbb9>:15: RunConfig.__init__ (from tensorflow.contrib.learn.python.learn.estimators.run_config) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.RunConfig instead.\n",
      "WARNING:tensorflow:From C:\\Users\\93246\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\estimators\\dnn.py:378: multi_class_head (from tensorflow.contrib.learn.python.learn.estimators.head) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.contrib.estimator.*_head.\n",
      "WARNING:tensorflow:From C:\\Users\\93246\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\estimators\\estimator.py:1179: BaseEstimator.__init__ (from tensorflow.contrib.learn.python.learn.estimators.estimator) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please replace uses of any Estimator from tf.contrib.learn with an Estimator from tf.estimator.*\n",
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x000002C2E08161D0>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1\n",
      "}\n",
      ", '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_session_config': None, '_save_checkpoints_steps': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 'models/model_DEEP_1560669305'}\n",
      "estimator built\n"
     ]
    }
   ],
   "source": [
    "def create_model_dir(model_type):\n",
    "    # Returns something like models/model_WIDE_AND_DEEP_1493043407\n",
    "    return 'models/model_' + model_type + '_' + str(int(time.time()))\n",
    "\n",
    "# Specify the desired model_dir \n",
    "def get_model(model_type, model_dir):\n",
    "    print(\"Model directory = %s\" % model_dir)\n",
    "    \n",
    "    # There are more options here than shown here. \n",
    "    # We are using this to show additional checkpointing for illustrative purposes.\n",
    "    # In a real system with far more samples, you would \n",
    "    #     likely choose to save checkpoints less frequently.\n",
    "    runconfig = tf.contrib.learn.RunConfig(\n",
    "        save_checkpoints_secs=None,\n",
    "        save_checkpoints_steps = 100,\n",
    "    )\n",
    "    \n",
    "    m = None\n",
    "    \n",
    "    # Linear Classifier\n",
    "    if model_type == 'WIDE':\n",
    "        m = tf.contrib.learn.LinearClassifier(\n",
    "            model_dir=model_dir, \n",
    "            feature_columns=wide_columns)\n",
    "\n",
    "    # Deep Neural Net Classifier\n",
    "    if model_type == 'DEEP':\n",
    "        m = tf.contrib.learn.DNNClassifier(\n",
    "            model_dir=model_dir,\n",
    "            feature_columns=deep_columns,\n",
    "            hidden_units=[100, 50, 25])\n",
    "\n",
    "    # Combined Linear and Deep Classifier\n",
    "    if model_type == 'WIDE_AND_DEEP':\n",
    "        m = tf.contrib.learn.DNNLinearCombinedClassifier(\n",
    "            model_dir=model_dir,\n",
    "            linear_feature_columns=wide_columns,\n",
    "            dnn_feature_columns=deep_columns,\n",
    "            dnn_hidden_units=[100, 70, 50, 25],\n",
    "            config=runconfig)\n",
    "        \n",
    "    print('estimator built')\n",
    "    \n",
    "    return m\n",
    "    \n",
    "\n",
    "MODEL_TYPE = 'DEEP'\n",
    "model_dir = create_model_dir(model_type=MODEL_TYPE)\n",
    "m = get_model(model_type=MODEL_TYPE, model_dir=model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载数据\n",
    "train_file = \"data/train.csv\"\n",
    "eval_file  = \"data/eval.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:From C:\\Users\\93246\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\estimators\\head.py:800: calling expand_dims (from tensorflow.python.ops.array_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the `axis` argument instead\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:From C:\\Users\\93246\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\estimators\\head.py:678: ModelFnOps.__new__ (from tensorflow.contrib.learn.python.learn.estimators.model_fn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.EstimatorSpec. You can use the `estimator_spec` method to create an equivalent one.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into models/model_DEEP_1560669305\\model.ckpt.\n",
      "INFO:tensorflow:loss = 126.36935, step = 1\n",
      "INFO:tensorflow:global_step/sec: 29.2666\n",
      "INFO:tensorflow:loss = 0.56949216, step = 101 (3.418 sec)\n",
      "INFO:tensorflow:global_step/sec: 69.2454\n",
      "INFO:tensorflow:loss = 0.566607, step = 201 (1.443 sec)\n",
      "INFO:tensorflow:global_step/sec: 69.2454\n",
      "INFO:tensorflow:loss = 0.5406747, step = 301 (1.447 sec)\n",
      "INFO:tensorflow:global_step/sec: 68.8176\n",
      "INFO:tensorflow:loss = 0.5690463, step = 401 (1.452 sec)\n",
      "INFO:tensorflow:global_step/sec: 68.9596\n",
      "INFO:tensorflow:loss = 0.63119346, step = 501 (1.449 sec)\n",
      "INFO:tensorflow:global_step/sec: 69.6301\n",
      "INFO:tensorflow:loss = 0.5122013, step = 601 (1.435 sec)\n",
      "INFO:tensorflow:global_step/sec: 68.5354\n",
      "INFO:tensorflow:loss = 0.56204826, step = 701 (1.460 sec)\n",
      "INFO:tensorflow:global_step/sec: 68.4419\n",
      "INFO:tensorflow:loss = 0.59979093, step = 801 (1.461 sec)\n",
      "INFO:tensorflow:global_step/sec: 57.5913\n",
      "INFO:tensorflow:loss = 0.55392134, step = 901 (1.736 sec)\n",
      "INFO:tensorflow:global_step/sec: 67.0773\n",
      "INFO:tensorflow:loss = 0.54949975, step = 1001 (1.492 sec)\n",
      "INFO:tensorflow:global_step/sec: 64.1915\n",
      "INFO:tensorflow:loss = 0.6161652, step = 1101 (1.557 sec)\n",
      "INFO:tensorflow:global_step/sec: 53.9072\n",
      "INFO:tensorflow:loss = 0.5754773, step = 1201 (1.858 sec)\n",
      "INFO:tensorflow:global_step/sec: 51.7642\n",
      "INFO:tensorflow:loss = 0.5210768, step = 1301 (1.932 sec)\n",
      "INFO:tensorflow:global_step/sec: 54.7609\n",
      "INFO:tensorflow:loss = 0.495078, step = 1401 (1.826 sec)\n",
      "INFO:tensorflow:global_step/sec: 54.6118\n",
      "INFO:tensorflow:loss = 0.5575, step = 1501 (1.830 sec)\n",
      "INFO:tensorflow:global_step/sec: 56.1093\n",
      "INFO:tensorflow:loss = 0.5212655, step = 1601 (1.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 58.0923\n",
      "INFO:tensorflow:loss = 0.5637251, step = 1701 (1.721 sec)\n",
      "INFO:tensorflow:global_step/sec: 57.7909\n",
      "INFO:tensorflow:loss = 0.56031054, step = 1801 (1.731 sec)\n",
      "INFO:tensorflow:global_step/sec: 57.891\n",
      "INFO:tensorflow:loss = 0.52927107, step = 1901 (1.723 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into models/model_DEEP_1560669305\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.57198846.\n",
      "fit done\n",
      "Wall time: 1min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# This can be found with\n",
    "# wc -l train.csv\n",
    "train_sample_size = 800000\n",
    "train_steps = train_sample_size/BATCH_SIZE # 8000/40 = 200\n",
    "\n",
    "m.fit(input_fn=generate_input_fn(train_file, BATCH_SIZE), steps=train_steps)\n",
    "\n",
    "print('fit done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 验证模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Rank of input Tensor (1) should be the same as output_rank (2) for column. Will attempt to expand dims. It is highly recommended that you resize your input, as this behavior may change.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Casting <dtype: 'int32'> labels to bool.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Starting evaluation at 2019-06-16-07:18:12\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from models/model_DEEP_1560669305\\model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Evaluation [50/500]\n",
      "INFO:tensorflow:Evaluation [100/500]\n",
      "INFO:tensorflow:Evaluation [150/500]\n",
      "INFO:tensorflow:Evaluation [200/500]\n",
      "INFO:tensorflow:Evaluation [250/500]\n",
      "INFO:tensorflow:Evaluation [300/500]\n",
      "INFO:tensorflow:Evaluation [350/500]\n",
      "INFO:tensorflow:Evaluation [400/500]\n",
      "INFO:tensorflow:Evaluation [450/500]\n",
      "INFO:tensorflow:Evaluation [500/500]\n",
      "INFO:tensorflow:Finished evaluation at 2019-06-16-07:18:21\n",
      "INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.753475, accuracy/baseline_label_mean = 0.251165, accuracy/threshold_0.500000_mean = 0.753475, auc = 0.56842387, auc_precision_recall = 0.38591185, global_step = 2000, labels/actual_label_mean = 0.251165, labels/prediction_mean = 0.254989, loss = 0.5511499, precision/positive_threshold_0.500000_mean = 0.59559125, recall/positive_threshold_0.500000_mean = 0.05755181\n",
      "evaluate done\n",
      "Accuracy: 0.753475\n",
      "{'loss': 0.5511499, 'accuracy': 0.753475, 'labels/prediction_mean': 0.254989, 'labels/actual_label_mean': 0.251165, 'accuracy/baseline_label_mean': 0.251165, 'auc': 0.56842387, 'auc_precision_recall': 0.38591185, 'accuracy/threshold_0.500000_mean': 0.753475, 'precision/positive_threshold_0.500000_mean': 0.59559125, 'recall/positive_threshold_0.500000_mean': 0.05755181, 'global_step': 2000}\n",
      "Wall time: 17.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "eval_sample_size = 200000 \n",
    "eval_steps = eval_sample_size/BATCH_SIZE \n",
    "\n",
    "results = m.evaluate(input_fn=generate_input_fn(eval_file), \n",
    "                     steps=eval_steps)\n",
    "print('evaluate done')\n",
    "\n",
    "print('Accuracy: %s' % results['accuracy'])\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "只用了DNN模型得到的结果   \n",
    "Accuracy: 0.753475   \n",
    "loss: 0.5511499   \n",
    "\n",
    "看起来效果比Linear模型还要差一点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
